package com.zlm.spider.processor;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zlm.spider.entity.Job;
import com.zlm.spider.pipeline.SpringDataPipeline;
import com.zlm.spider.util.SalaryUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.DuplicateRemovedScheduler;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;

import java.util.HashMap;
import java.util.Map;

/**
 * @author Harbour
 * @date 2021-04-13 9:27
 */
@Component
public class JobProcessor implements PageProcessor {

    private static int totalTimes = 0;

    @Autowired
    private SpringDataPipeline springDataPipeline;

    @Autowired
    ConfigurableApplicationContext context;

    private final Map<String, Map<String, String>> jobInfoMap = new HashMap<>(); // 保存jobId 与 jobInfo的映射

    private final Site site = Site.me()
            .setUserAgent("Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0")
            .setCharset("GBK")  // 设置编码
            .setTimeOut(1000 * 10) // 设置超时时间
            .setRetrySleepTime(3000) // 设置重试超时时间
            .setRetryTimes(3); // 设置重试次数

    @Scheduled(initialDelay = 1000, fixedDelay = 1)
    public void process() {
        totalTimes++;

        if (totalTimes > 2000) {
//            context.close();
            int exitCode = SpringApplication.exit(context);
            System.exit(exitCode);
        }

        String url = "https://search.51job.com/list/000000,000000,0000,32%252c01,9,99,+,2," + totalTimes +
                ".html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99" +
                "&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
        DuplicateRemovedScheduler scheduler = new QueueScheduler()
                .setDuplicateRemover(new BloomFilterDuplicateRemover(1000 * 1000));

        Spider.create(new JobProcessor())
                .addUrl(url)
                .addPipeline(springDataPipeline)
                .setScheduler(scheduler)
                .thread(10)
                .run();
    }

    @Override
    public void process(Page page) {

        // todo : 1. 获取html页面
        Html html = page.getHtml();

        // todo : 2. 解析json对象
        String jobListStr = html.xpath("/html/body/script[2]").get();

        // todo : 3. 保存 Job 信息
        if (!jobListStr.contains("window.__SEARCH_RESULT__ = ")) {
            // jsonArray为空，说明这个页面是job详情页
            saveJobInfo(page, jobInfoMap);
        } else {
            String jsonStr = jobListStr.replace("<script type=\"text/javascript\">", "").replace("window.__SEARCH_RESULT__ = ", "")
                    .replace("\\","").split(",\"jobid_count")[0]+ "}";
            JSONArray jsonArray = JSON.parseArray(JSON.parseObject(jsonStr).getString("engine_search_result"));
            for (Object o : jsonArray) {
                JSONObject jsonObject = (JSONObject) o;

                String jobHref = jsonObject.getString("job_href");

                Map<String, String> jobInfo = new HashMap<>();
                jobInfo.put("jobid", jsonObject.getString("jobid"));
                jobInfo.put("providesalary_text", jsonObject.getString("providesalary_text"));
                jobInfo.put("workarea_text", jsonObject.getString("workarea_text"));
                jobInfo.put("companysize_text", jsonObject.getString("companysize_text"));
                jobInfo.put("company_name", jsonObject.getString("company_name"));
                jobInfo.put("issuedate", jsonObject.getString("issuedate"));
                jobInfo.put("job_name", jsonObject.getString("job_name"));
                jobInfoMap.put(jobHref, jobInfo);

                page.addTargetRequest(jobHref);
            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 装配 job 对象
     * @param page 传进来网页
     */
    private void saveJobInfo(Page page, Map<String, Map<String, String>> jobInfoMap) {
        Job job = new Job();

        Html html = page.getHtml();

        String pageURL = page.getUrl().toString();

        // 一系列xpath和字符串替换的操作
        String s = html.xpath("/html/body/div[3]/div[2]/div[3]/div[1]/div").toString();

        String jobInfo = "";
        if (s != null && s.length() > 0) {
            jobInfo = replaceStr(s).split("<div class=\"mt10\">")[0]
                    .replace("<div class=\"bmsg job_msg inbox\">", "");
        }

        String c = html.xpath("/html/body/div[3]/div[2]/div[3]/div[3]/div/text()").get();
        String compInfo = "";
        if (c != null && c.length() > 0) {
            compInfo = replaceStr(c);
        }

        String salaryStr = jobInfoMap.get(pageURL).get("providesalary_text");
        Integer[] salary = SalaryUtils.getSalary(salaryStr);

        String addrStr = jobInfoMap.get(pageURL).get("workarea_text");

        job.setCompanyName(jobInfoMap.get(pageURL).get("company_name"));
        job.setCompanyAddr(addrStr);
        job.setCompanyInfo(compInfo);

        job.setJobInfo(jobInfo);
        job.setJobAddr(html.xpath("/html/body/div[3]/div[2]/div[3]/div[2]/div/p/text()").get());
        job.setJobName(jobInfoMap.get(pageURL).get("job_name"));

        job.setTime(jobInfoMap.get(pageURL).get("issuedate"));
        job.setUrl(pageURL);

        job.setSalaryMax(salary[1]);
        job.setSalaryMin(salary[0]);

        page.putField("job", job);
    }

    private String replaceStr(String str) {
        return str.replace("&nbsp;", "")
                .replace("<p>", "")
                .replace("</p>", "")
                .replace("<div>", "")
                .replace("</div>", "")
                .replace("<br>", "")
                .replace("<br>", "");
    }
}
